# 通过非阻塞io实现http请求
# select + 回调 + 事件循环
# 并发性高
import socket
from urllib.parse import urlparse
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE


selector = DefaultSelector()
# 使用select完成http请求
urls = ['http://www.baidu.com']
stop = False

class Fetcher:

    def connected(self, key):
        selector.unregister(key.fd)
        self.client.send('GET {} HTTP/1.1\r\n Host:{}\r\nConnection :close\r\n\r\n'.format(self.path, self.host).encode('utf8'))
        selector.register(self.client.fileno(), EVENT_READ, self.readable )

    def readable(self, key):

        d = self.client.recv(1024)
        if d:
            print(d)
            self.data += d
        else:
            selector.unregister(key.fd)
            data = self.data.decode('utf8')
            html_data = data.split('\r\n\r\n')[1]
            print(html_data)
            self.client.close()
            urls.remove(self.spider_url)
            if not urls:
                global stop
                stop = True

    def get_url(self, url):
        self.spider_url = url
        print(url)
        url = urlparse(url)
        self.host = url.netloc
        self.path = url.path
        self.data = b''
        if self.path == '':
            self.path = '/'

        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        self.client.setblocking(False)
        try:
            self.client.connect((self.host, 80))
        except BlockingIOError as e:
            pass

#         注册
        selector.register(self.client.fileno(), EVENT_WRITE, self.connected)


def loop():
#     时间循环，不停的请求soket状态并调用相应的回调函数
#     select 本身是不支持register模式
#     socket状态变化后的回调是程序员完成的
    while not stop:
        ready = selector.select()
        for key, mask in ready:
            call_back = key.data
            call_back(key)
#         回调+事件循环+select(poll\epoll)


# def get_url(url):
#     url = urlparse(url)
#     host = url.netloc
#     path = url.path
#     if path == '':
#         path = '/'
#
#     client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#
#     client.setblocking(False)
#
#     try:
#         client.connect((host, 80))
#     except BlockingIOError as e:
#         pass
#
#     while True:
#         try:
#             client.send('GET {} HTTP/1.1\r\n Host:{}\r\nConnection :close\r\n\r\n'.format(path, host).encode('utf8'))
#             break
#         except OSError as e:
#             pass
#
#     data = b''
#     while True:
#         try:
#             d = client.recv(1024)
#         except BlockingIOError as e:
#             continue
#         if d:
#             data += d
#         else:
#             break
#
#     data = data.decode('utf8')
#     html_data = data.split('\r\n\r\n')[1]
#     print(html_data)
#     client.close()


if __name__ == '__main__':
    # import time
    # start_time = time.time()
    # for url in range(20):
    #     url = 'http://shop.projectsedu.com/goods/{}/'.format(url)
    #     get_url(url)
    # print(time.time() - start_time)

    fetcher = Fetcher()
    fetcher.get_url('http://www.baidu.com')
    loop()